"""
教育数据加载模块（适配清洗后数据版）

主要改进：
1. 完全适配清洗后的数据字段结构
2. 优化日期时间处理逻辑
3. 增强类型转换的健壮性
4. 添加数据验证步骤
"""
import numpy as np  # 添加这行
import pandas as pd
from pathlib import Path
from typing import Dict, Optional
import chardet
import logging
from datetime import datetime


class EduDataLoader:
    """教育数据加载器（适配清洗后数据结构）"""

    # === 路径配置区 ===
    BASE_DATA_DIR = Path(r"D:\code\python\commercial\clean")  # 清洗后数据目录
    BASE_LOG_DIR = Path(r"D:\code\python\commercial\log")    # 日志目录

    # 清洗后的文件名定义
    FILE_NAMES = {
        'teacher': 'cleaned_teachers.csv',
        'student': 'cleaned_students.csv',
        'attendance': 'cleaned_attendance.csv',
        'attendance_type': 'cleaned_kaoqin_type.csv',
        'score': 'cleaned_grades.csv',
        'consumption': 'cleaned_consumption.csv'
    }
    LOG_FILE_NAME = 'data_loader.log'
    # === 路径配置结束 ===

    # 清洗后数据结构定义
    DATA_SCHEMA = {
        'student': ['bf_StudentID', 'bf_Name', 'bf_sex', 'age', 'cla_Name', 'policy_code', 'dorm_status', 'is_local'],
        'teacher': ['term', 'cla_Name', 'teacher_name', 'subject', 'start_year', 'semester'],
        'attendance': ['bf_studentID', 'datetime', 'date', 'hour', 'day_of_week', 'is_weekday', 'time_slot', 'std_type', 'severity'],
        'attendance_type': ['control_task_order_id', 'std_type', 'severity'],
        'score': ['bf_studentID', 'exam_date', 'score', 'score_status', 'subject', 'EXAM_KIND_NAME'],
        'consumption': ['bf_StudentID', 'datetime', 'amount', 'period']
    }

    def __init__(self):
        """初始化加载器"""
        # 构建完整路径
        self.DATA_PATHS = {
            name: self.BASE_DATA_DIR / fname
            for name, fname in self.FILE_NAMES.items()
        }
        self.LOG_PATH = self.BASE_LOG_DIR / self.LOG_FILE_NAME

        self._init_logger()
        self._validate_paths()
        self.logger.info(f"数据目录: {self.BASE_DATA_DIR}")
        self.logger.info(f"日志目录: {self.BASE_LOG_DIR}")

    def _init_logger(self):
        """初始化日志系统"""
        self.BASE_LOG_DIR.mkdir(parents=True, exist_ok=True)

        self.logger = logging.getLogger('EduDataLoader')
        self.logger.setLevel(logging.INFO)

        # 文件日志
        file_handler = logging.FileHandler(self.LOG_PATH, encoding='utf-8')
        file_handler.setFormatter(
            logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
        )

        # 控制台日志
        console_handler = logging.StreamHandler()
        console_handler.setFormatter(
            logging.Formatter('%(levelname)s - %(message)s')
        )

        self.logger.addHandler(file_handler)
        self.logger.addHandler(console_handler)

    def _validate_paths(self):
        """验证所有数据文件是否存在"""
        missing_files = []
        for name, path in self.DATA_PATHS.items():
            if not path.exists():
                missing_files.append(f"{name}: {path}")

        if missing_files:
            error_msg = "数据文件缺失:\n" + "\n".join(missing_files)
            self.logger.error(error_msg)
            raise FileNotFoundError(error_msg)

    def _load_file(self, file_type: str) -> pd.DataFrame:
        """加载单个数据文件（适配清洗后结构）"""
        file_path = self.DATA_PATHS[file_type]
        self.logger.info(f"加载文件: {file_path.name}")

        # 编码检测与处理
        try:
            with open(file_path, 'rb') as f:
                rawdata = f.read(10000)
                encoding = chardet.detect(rawdata)['encoding']
                encoding = 'gbk' if encoding and 'gb' in encoding.lower() else (encoding or 'utf-8')
        except Exception as e:
            self.logger.error(f"编码检测失败: {str(e)}")
            raise

        # 动态加载参数（适配清洗后结构）
        load_params = {
            'student': {
                'dtype': {
                    'bf_StudentID': 'int32',
                    'bf_sex': 'category',
                    'policy_code': 'int8',
                    'is_local': 'int8'
                },
                'parse_dates': []
            },
            'teacher': {
                'dtype': {
                    'start_year': 'int16',
                    'semester': 'category'
                }
            },
            'attendance': {
                'parse_dates': ['datetime', 'date'],
                'dtype': {
                    'hour': 'int8',
                    'day_of_week': 'int8',
                    'is_weekday': 'int8',
                    'severity': 'int8'
                }
            },
            'score': {
                'parse_dates': ['exam_date'],
                'dtype': {
                    'score': 'float32',
                    'score_status': 'category'
                }
            },
            'consumption': {
                'parse_dates': ['datetime'],
                'dtype': {
                    'amount': 'float32',
                    'period': 'category'
                }
            }
        }

        try:
            df = pd.read_csv(
                file_path,
                encoding=encoding,
                **load_params.get(file_type, {})
            )

            # 验证必要字段
            self._validate_schema(df, file_type)

            # 数据优化
            df = self._optimize_data(df, file_type)

            self.logger.info(f"成功加载 {file_path.name} [记录数: {len(df)}]")
            return df
        except Exception as e:
            self.logger.error(f"加载失败: {str(e)}")
            raise

    def _validate_schema(self, df: pd.DataFrame, file_type: str):
        """验证数据字段是否符合预期"""
        required_cols = self.DATA_SCHEMA.get(file_type, [])
        missing_cols = [col for col in required_cols if col not in df.columns]

        if missing_cols:
            error_msg = f"{file_type}数据缺少必要列: {missing_cols}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    def _optimize_data(self, df: pd.DataFrame, file_type: str) -> pd.DataFrame:
        """数据优化处理（适配清洗后结构）"""
        # 通用优化
        for col in df.select_dtypes(include=['object']):
            if df[col].nunique() / len(df) < 0.5:
                df[col] = df[col].astype('category')

        # 特定优化
        if file_type == 'student':
            if 'age' in df.columns:
                df['age'] = pd.to_numeric(df['age'], errors='coerce').fillna(0).astype('int8')

        elif file_type == 'score':
            if 'score' in df.columns:
                df['score'] = df['score'].clip(0, 100)  # 确保分数在合理范围

        elif file_type == 'consumption':
            if 'amount' in df.columns:
                df['amount'] = df['amount'].abs()  # 确保金额为正

        return df

    # 各数据加载方法
    def load_students(self) -> pd.DataFrame:
        """加载学生信息"""
        df = self._load_file('student')

        # 后处理
        if 'dorm_status' in df.columns:
            df['dorm_status'] = df['dorm_status'].astype('category')

        return df

    def load_teachers(self) -> pd.DataFrame:
        """加载教师信息"""
        return self._load_file('teacher')

    def load_attendance(self) -> pd.DataFrame:
        """加载考勤记录"""
        return self._load_file('attendance')

    def load_scores(self) -> pd.DataFrame:
        """加载成绩数据"""
        df = self._load_file('score')

        # 处理缺考记录
        if 'score_status' in df.columns:
            df.loc[df['score_status'] == '缺考', 'score'] = np.nan

        return df

    def load_consumption(self) -> pd.DataFrame:
        """加载消费数据"""
        df = self._load_file('consumption')

        # 添加日期列
        if 'datetime' in df.columns:
            df['date'] = df['datetime'].dt.date

        return df

    def load_all(self) -> Dict[str, pd.DataFrame]:
        """一键加载所有数据集"""
        return {
            'students': self.load_students(),
            'teachers': self.load_teachers(),
            'attendance': self.load_attendance(),
            'scores': self.load_scores(),
            'consumption': self.load_consumption()
        }

# 使用示例
if __name__ == "__main__":
    print("=== 教育数据加载器使用说明 ===")
    print("修改以下变量后使用:")
    print("1. BASE_DATA_DIR - 清洗后数据存放目录")
    print("2. BASE_LOG_DIR - 日志文件存放目录\n")

    try:
        loader = EduDataLoader()
        data = loader.load_all()

        print("\n数据加载成功！")
        print("各数据集记录数:")
        for name, df in data.items():
            print(f"{name:>12}: {len(df):<6}条记录")
            print(f"字段示例: {list(df.columns)[:3]}...")

    except Exception as e:
        print(f"\n错误: {str(e)}")
        print("请检查:")
        print(f"1. 数据目录是否存在: {EduDataLoader.BASE_DATA_DIR}")
        print(f"2. 文件命名是否正确: {EduDataLoader.FILE_NAMES}")
        print(f"3. 查看详细日志: {EduDataLoader.LOG_PATH}")